﻿u	sing System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Resources;
using System.Reflection;
using System.Globalization;
using System.Threading;

namespace ExchangeHog.MessageParser
{
	public class LexerException: Exception
	{
		public LexerException(string message): base(message) {}
	}

	public class Token
	{

		public enum TokenType
		{
			TOKEN_NONE, //TOKEN_CRLF,
			CHN_HEADERNAME, CHN_COLON, CHN_EMPTYLINE,
			CAHV_ATOM, CAHV_QUOTED_STRING, CAHV_PERIOD, CAHV_AT, CAHV_DOMAIN_LITERAL, CAHV_ROUTING_INFORMATION, CAHV_COLON, CAHV_SEMICOLON, CAHV_COMMA, CAHV_LWS,
			CHP_SLASH, CHP_TOKEN, CHP_SEMICOLON, CHP_QUOTEDSTRING, CHP_EQUAL, CHP_CRLF,
			CMHV_MIMEVERSION_VALUE,
			CUWEW_TEXT, cUWEW_CRLF,
			CMB_FINAL_BOUNDARY, CMB_EOF, CMB_BOUNDARY,
			CU_TEXT, CU_CRLF,
			CUWEW_CRLF,
			CAHV_ENCODED_WORD,
			CAHV_WORD,
			CAHV_LEFT_ANGLE_BRACKET
		};
		public TokenType TType { get; set; }

		public string Repr { get; set; }

		public Token(TokenType ttype) { TType = ttype; }
		public Token(TokenType ttype, string repr) { TType = ttype; Repr = repr;}
	}

	public class Lexer
	{
		public enum ContextType
		{
			CONTEXT_NONE, 
			HEADER_FIELD, // ... :
			CONTEXT_ADDRESS_HEADER_VALUE, // from, to, etc : ...
			CONTEXT_CONTENT_HEADER_PARAM, // content-type: ...
			CONTEXT_MIMEVERSION_HEADER_VALUE, // mime-version: ...
			CONTEXT_UNSTRUCTUREDTEXT_WITH_ENCODED_WORDS, // subjects, comments: ...
			CONTEXT_MESSAGE_BODY, // body all the way to the end or to the boundary(--)
			CONTEXT_UNSTRUCTUREDTEXT // <some_header> : ...
		};
		public ContextType Context { public get; public set; }

		StreamReader sr;

		List<Char> lookaheadChars;
		Char PopFirstFromLookaheadList() { if (lookaheadChars.Count < 1) throw new ArgumentException(); Char ch = lookaheadChars[0]; lookaheadChars.RemoveAt(0); return ch; }
		Char PeekFirstFromLookaheadList() { if (lookaheadChars.Count < 1) throw new ArgumentException(); return lookaheadChars[0]; }

		ResourceManager rm;

		public Lexer(Stream s, ContextType initialcontext)
		{
			lookaheadChars = new List<char>();
			lookaheadTokens = new Queue<Tuple<Token, ContextType>>();
			sr = new StreamReader(s, Encoding.ASCII);
			rm = new ResourceManager(typeof(Lexer));
			Context = initialcontext;
		}
		private string GetMessage(string resourcename, params object[] args) { return String.Format(rm.GetString(resourcename, Thread.CurrentThread.CurrentCulture), args); }


		private Char ReadChar() { return lookaheadChars.Count > 0? PopFirstFromLookaheadList(): (char)sr.Read(); }
		private Char PeekChar() { return lookaheadChars.Count > 0 ? PeekFirstFromLookaheadList() : (char)sr.Peek(); }
		private Char PeekChar2()
		{
			if (lookaheadChars.Count < 1) lookaheadChars.Add(ReadChar());
			if (lookaheadChars.Count < 2) lookaheadChars.Add(ReadChar());
			return Convert.ToChar(lookaheadChars[1]);
		}
		private Char PeekChar3()
		{
			if (lookaheadChars.Count < 1) lookaheadChars.Add(ReadChar());
			if (lookaheadChars.Count < 2) lookaheadChars.Add(ReadChar());
			if (lookaheadChars.Count < 3) lookaheadChars.Add(ReadChar());
			return Convert.ToChar(lookaheadChars[2]);
		}

		Queue<Tuple<Token, ContextType>> lookaheadTokens;

		private void SkipLWS()
		{
			Char ch = ReadChar();
			if (ch != ' ' && ch != '\t')
				throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "white space(space or tab)"));
			if (!sr.EndOfStream)
			{
				bool isEndOfLWS = false;
				ch = PeekChar();
				while (!sr.EndOfStream && !isEndOfLWS)
					switch (ch)
					{
						case ' ': case '\t': ReadChar(); break;
						case '\r':
							if (PeekChar2() == '\n' && (PeekChar3() == ' ' || PeekChar3() == '\t'))
							{
								ReadChar(); // \r
								ReadChar(); // \n
								ReadChar(); // ' ' or '\t'
							}
							ch = PeekChar();
							break;
						default:
							isEndOfLWS = true;
							break;
					}
			}
		} // end of SkipLWS

		private Token ParseComment(Token.TokenType tt)
		{
			Char ch = ReadChar();
			if (ch != '(')
				throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "left parenthesis"));

			StringBuilder sb = new StringBuilder();
			int depth = 1;
			bool isEndOfCommentReached = false;
			while (!sr.EndOfStream && !isEndOfCommentReached)
			{
				ch = ReadChar();
				switch (ch)
				{
					case ')':
						if ((--depth) == 0)
							isEndOfCommentReached = true;
						break;
					case '(':
						depth++;
						break;
					case '\\': // quoted character
						ch = ReadChar();
						if (ch == '\r')
						{
							ch = PeekChar();
							if (ch == '\n')
							{
								sb.Append("\n"); // CRLF is quoted - add it to the comment's body
								ReadChar();
								SkipLWS();
							}
						}
						else
							sb.Append(ch);
						break;
					default:
						sb.Append(ch);
						break;
				}
			}
			return new Token(tt, sb.ToString());
		} // end of ParseComment

		//
		// Angle brackets ("<" and ">")  are  generally  used  to
		// indicate  the  presence of a one machine-usable refer-
		// ence (e.g., delimiting mailboxes), possibly  including
		// source-routing to the machine.
		//
		private Token ParseRoutingInformation()
		{
			Char ch = ReadChar();
			if (ch != '<')
				throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "left angle bracket"));

			StringBuilder sb = new StringBuilder();
			bool isEndOfRoutingInformationReached = false;
			while (!sr.EndOfStream && !isEndOfRoutingInformationReached)
			{
				ch = ReadChar();
				switch (ch)
				{
					case ' ': case '\t': SkipLWS(); break;
					case '\\': // quoted character
						ch = ReadChar();
						if (ch == '\r')
						{
							ch = PeekChar();
							if (ch == '\n')
							{
								sb.Append("\n"); // CRLF is quoted - add it to the domain literal
								ReadChar();
								SkipLWS();
							}
						}
						else
							sb.Append(ch);
						break;
					case '>': isEndOfRoutingInformationReached = true; break;
					default:
						sb.Append(ch);
						break;
				}
			}
			return new Token(Token.TokenType.CAHV_ROUTING_INFORMATION, sb.ToString());
		} // end of ParseRoutingInformation

		//
		// Square brackets ("[" and "]") are used to indicate the
		// presence  of  a  domain-literal, which the appropriate
		// name-domain  is  to  use  directly,  bypassing  normal
		// name-resolution mechanisms.
		//
		private Token ParseDomainLiteral()
		{
			Char ch = ReadChar();
			if (ch != '[')
				throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "left square bracket"));

			StringBuilder sb = new StringBuilder();
			bool isEndOfDomainLiteralReached = false;
			while (sr.EndOfStream && !isEndOfDomainLiteralReached)
			{
				ch = ReadChar();
				switch (ch)
				{
					case ' ': case '\t': SkipLWS(); break;
					case ']': isEndOfDomainLiteralReached = true; break;
					case '\\': // quoted character
						ch = ReadChar();
						if (ch == '\r')
						{
							ch = PeekChar();
							if (ch == '\n')
							{
								sb.Append("\n"); // CRLF is quoted - add it to the domain literal
								ReadChar();
								SkipLWS();
							}
						}
						else
							sb.Append(ch);
						break;
					default:
						sb.Append(ch);
						break;
				}
			}
			return new Token(Token.TokenType.CAHV_DOMAIN_LITERAL, sb.ToString());
		} // end of ParseDomainLiteral

		//	CHAR        =  <any ASCII character>        ; (  0-177,  0.-127.)
		//	CTL         =  <any ASCII control           ; (  0- 37,  0.- 31.)
		//	                character and DEL>          ; (    177,     127.)
		//	SPACE       =  <ASCII SP, space>            ; (     40,      32.)
		//	specials    =  "(" / ")" / "<" / ">" / "@"  ; Must be in quoted-
		//	            /  "," / ";" / ":" / "\" / <">  ;  string, to use
		//	            /  "." / "[" / "]"              ;  within a word.
		//	atom        =  1*<any CHAR except specials, SPACE and CTLs> /
		//					encoded-word				; from RFC 2047 (see below)
		//
		//		from RFC 2047
		//
		//	encoded-word = "=?" charset "?" encoding "?" encoded-text "?="
		//	charset = token    ; see section 3
		//	encoding = token   ; see section 4
		//	token = 1*<Any CHAR except SPACE, CTLs, and especials>
		//	especials = specials / "?" / "="
		//	encoded-text = 1*<Any printable ASCII character other than "?" or SPACE>
		//

		static bool DoesBelongToAtom(Char ch)
		{
			return
				31 < ((short)ch) && ((short)ch) < 127 &&
				ch != ' ' && ch != '\t' &&
				ch != '(' && ch != ')' && ch != '<' && ch != '>' && ch != '@' &&
				ch != ',' && ch != ';' && ch != ':' && ch != '\\' && ch != '\"' &&
				ch != '.' && ch != '[' && ch != ']';
		}

		static bool DoesBelongToEncodedText(Char ch)
		{
			return DoesBelongToAtom(ch) && ch != '?' && ch != '=';
		}

		private String ParseEToken()
		{
			StringBuilder sb = new StringBuilder();
			Char ch = PeekChar();
			while (!sr.EndOfStream && DoesBelongToEncodedText(ch))
			{
				ReadChar();
				sb.Append(ch);
			}
			return sb.ToString();
		}

		private Token ParseAtom()
		{
			String atomrepr;

			Char ch = ReadChar();
			if (ch == '=' && PeekChar() == '?')
			{
				//
				//	encoded word
				//
				ReadChar(); // skip '?'
				//
				//	charset
				//
				string charset = ParseEToken();
				ch = ReadChar();
				if (ch != '?')
					throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "question mark"));
				//
				//	encoding
				//
				string encoding = ParseEToken();
				ch = ReadChar();
				if (ch != '?')
					throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "question mark"));

				StringBuilder sb = new StringBuilder();
				ch = ReadChar();
				while (!sr.EndOfStream && ch != '?' && ch != ' ')
				{
					sb.Append(ch);
					ch = ReadChar();
				}

				if (ch != '?')
					throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "question mark"));
				ch = ReadChar();
				if (ch != '?')
					throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "question mark"));

				atomrepr = Decode(sb.ToString(), charset, encoding);
			}
			else
			{
				if (!DoesBelongToAtom(ch))
					throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "any char except specials, space and ctls"));

				StringBuilder sb = new StringBuilder();
				sb.Append(ch);

				if (!sr.EndOfStream)
				{
					ch = PeekChar();

					while (!sr.EndOfStream && DoesBelongToAtom(ch))
					{
						sb.Append(ch);
						ReadChar();
						if (!sr.EndOfStream)
							ch = PeekChar();
					}
				}
				atomrepr = sb.ToString();
			}
			return new Token(Token.TokenType.CAHV_ATOM, atomrepr);
		} // end of ParseAtom

		private static Char DecodeQuotedPrintableChar(string s, Decoder decoder)
		{
			byte code = Byte.Parse(s.Substring(1, 2));
			char[] tchars = new char[1];
			decoder.GetChars(new byte[] { code }, 0, 1, tchars, 0, true);
			return tchars[0];
		}

		public string Decode(string s, string charset, string encoding)
		{
			string result;
			if (String.Compare(encoding, "q", true) == 0) // compare ignoring case
			{
				//
				//	QUOTED-PRINTABLE encoding
				//
				Decoder decoder = Encoding.GetEncoding(charset).GetDecoder();

				StringBuilder sb = new StringBuilder();
				int lastpeq = 0;
				int peq = s.IndexOf('=');
				while (peq >= 0)
				{
					sb.Append(s.Substring(lastpeq, peq - lastpeq + 1));
					sb.Append(DecodeQuotedPrintableChar(s.Substring(peq, 3), decoder));
					lastpeq = peq;
					if (lastpeq < s.Length)
						peq = s.IndexOf('=', lastpeq + 1);
					else
						peq = -1;
				}
				if (peq < s.Length - 1)
					sb.Append(s.Substring(peq + 1));
				result = sb.ToString();
			}
			else if (String.Compare(encoding, "b", true) == 0)
			{
				//
				//	BASE64 encoding
				//
				try
				{
					System.Text.Decoder utf8decoder = Encoding.GetEncoding(charset).GetDecoder();

					byte[] sbytes = Convert.FromBase64String(s);
					int tcharscount = utf8decoder.GetCharCount(sbytes, 0, sbytes.Length);
					char[] tchars = new char[tcharscount];
					utf8decoder.GetChars(sbytes, 0, sbytes.Length, tchars, 0);
					result = new String(tchars);
				}
				catch (Exception e)
				{
					throw new LexerException(GetMessage("LexerError_FailedToDecodeBase64Content", e.Message));
				}

			}
			else
				throw new LexerException(GetMessage("LexerError_UnsupportedEncoding", encoding));
			return result;
		} // end of Decode

		protected void ParseCRLF()
		{
			Char ch = ReadChar();
			Expect(delegate() { return (ch == '\r'); }, ch, "CR");
			ch = ReadChar();
			Expect(delegate() { return (ch == '\n'); }, ch, "LF");

			Token t = new Token();
			t.MarkAsCRLF();
			Tokens.Enqueue(t);
		} // end of ParseCRLF

		private Token ParseQuotedString(Token.TokenType tt)
		{
			Char ch = ReadChar();
			if (ch != '\"')
				throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "quote mark"));

			StringBuilder sb = new StringBuilder();
			bool isEndOfQuotedStringReached = false;
			while (!sr.EndOfStream && !isEndOfQuotedStringReached)
			{
				ch = ReadChar();
				switch (ch)
				{
					case '\"':
						isEndOfQuotedStringReached = true;
						break;
					case '\\':
						ch = ReadChar();
						if (ch == '\r')
						{
							ch = PeekChar();
							if (ch == '\n')
							{
								sb.Append("\n"); // CRLF is quoted - add it to the string's body
								ReadChar();
								SkipLWS();
							}
						}
						else
							sb.Append(ch);
						break;
					default:
						sb.Append(ch);
						break;
				}
			}
			return new Token(tt, sb.ToString());
		} // end of ParseQuotedString

		//
		//	tspecials :=  "(" / ")" / "<" / ">" / "@" /
		//	              "," / ";" / ":" / "\" / <">
		//	              "/" / "[" / "]" / "?" / "="
		//
		static bool DoesBelongToContentHeaderToken(Char ch)
		{
			return
				31 < ((short)ch) && ((short)ch) < 127 &&
				ch != ' ' && ch != '\t' &&
				ch != '(' && ch != ')' && ch != '<' && ch != '>' && ch != '@' &&
				ch != ',' && ch != ';' && ch != ':' && ch != '\\' && ch != '\"' &&
				ch != '/' && ch != '[' && ch != ']' && ch != '?' && ch != '=';
		}

		private Token ParseContentHeaderToken()
		{
			String repr;

			Char ch = ReadChar();
			if (!DoesBelongToContentHeaderToken(ch))
				throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "any char except specials, space and ctls"));

			StringBuilder sb = new StringBuilder();
			sb.Append(ch);

			if (!sr.EndOfStream)
			{
				ch = PeekChar();
				while (!sr.EndOfStream && DoesBelongToContentHeaderToken(ch))
				{
					sb.Append(ch);
					ReadChar();
					ch = PeekChar();
				}
			}
			repr = sb.ToString();
			return new Token(Token.TokenType.CHP_TOKEN, repr);
		} // end of ParseContentHeaderToken

		public Token NextToken(ContextType newcontext = ContextType.CONTEXT_NONE)
		{
			if (newcontext != ContextType.CONTEXT_NONE)
				Context = newcontext;

			Token t = null;
			if (lookaheadTokens.Count > 0)
			{
				Tuple<Token, ContextType> tuple = lookaheadTokens.Dequeue();
				if (tuple.Item2 != Context)
					throw new LexerException(GetMessage("LexerError_AttemptToProcessLookaheadTokensRetrievedOnDifferentContext", tuple.Item1.TType, Context, tuple.Item2));
				t = tuple.Item1;
			}
			else
				switch (Context)
				{
					case ContextType.HEADER_FIELD: // ... :
					{
						bool areDone = false;
						StringBuilder sb = new StringBuilder();
						while (!areDone && !sr.EndOfStream)
						{
							char ch = ReadChar();
							switch (ch)
							{
								case ':': t = new Token(Token.TokenType.CHN_COLON); areDone = true; break;
								case ' ': case '\t': SkipLWS(); break;
								case '\r': 
									if (PeekChar() == '\n') 
									{
										t = new Token(Token.TokenType.CHN_EMPTYLINE); ReadChar(); areDone = true;
									}
									else
										throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "empty line"));
									break;
								default:
									if (((int)ch) > 32 && ((int)ch) < 127)
										while(true)
										{
											sb.Append(ch);
											ch = PeekChar();
											if (!sr.EndOfStream && ((int)ch) > 32 && ((int)ch) < 127)
												ReadChar();
											else 
												break;
										}
									else
										throw new LexerException(GetMessage("LexerError_UnexpectedCharacterButExpected", ch, "any character but space and ctls"));
									break;
							}
						}
						if (t == null)
							t = new Token(Token.TokenType.CHN_HEADERNAME, sb.ToString());
						break;
					}
					case ContextType.UNSTRUCTURED_HEADER_FIELD_BODY:
					{
						bool areDone = false;
						StringBuilder sb = new StringBuilder();
						while (!areDone && !sr.EndOfStream)
						{
							char ch = ReadChar();
							switch (ch)
							{
								case '\r': areDone = true; break;
								case '\t': case ' ': SkipLWS(); break;
								default: sb.Append(ch);
							}
							if (t != null)
								areDone = true;
						}
						break;
					}



					case ContextType.CONTEXT_ADDRESS_HEADER_VALUE: // from, to, etc : ...
					{
						bool areDone = false;
						StringBuilder sb = new StringBuilder();
						while (!areDone && !sr.EndOfStream)
						{
							char ch = PeekChar();
							switch (ch)
							{
								case '\r': areDone = true; break;
								case '@': ReadChar(); t = new Token(Token.TokenType.CAHV_AT); break;
								case '.': ReadChar(); t = new Token(Token.TokenType.CAHV_PERIOD); break;
								case ',': ReadChar(); t = new Token(Token.TokenType.CAHV_COMMA); break;
								case ':': ReadChar(); t = new Token(Token.TokenType.CAHV_COLON); break;
								case ';': ReadChar(); t = new Token(Token.TokenType.CAHV_SEMICOLON); break;
								case '(': ParseComment(Token.TokenType.CAHV_LWS); break;
								case '[': t = ParseDomainLiteral(); break;
								case '<': t = ParseRoutingInformation(); break;
								case '"': t = ParseQuotedString(Token.TokenType.CAHV_QUOTED_STRING); break;
								case '\t': case ' ': SkipLWS(); break;
								default:
									if (0 < (short)ch && (short)ch < 127)
										t = ParseAtom();
									else
										throw new LexerException(GetMessage("LexerError_UnexpectedCharacter", ch));
									break;
							}
							if (t != null)
								areDone = true;
						}
						break;
					}
					case ContextType.CONTEXT_CONTENT_HEADER_PARAM: // content-type: ...
					{
						bool areDone = false;
						StringBuilder sb = new StringBuilder();
						while (!areDone && !sr.EndOfStream)
						{
							char ch = ReadChar();
							switch (ch)
							{
								case '\r': areDone = true; break;
								case '=': ReadChar(); t = new Token(Token.TokenType.CHP_EQUAL); break;
								case ';': ReadChar(); t = new Token(Token.TokenType.CHP_SEMICOLON); break;
								case '"': t = ParseQuotedString(Token.TokenType.CHP_QUOTEDSTRING); break;
								case '\t': case ' ': SkipLWS(); break;
								default: t = ParseContentHeaderToken(); break;
							}
							if (t != null)
								areDone = true;
						}
						break;
					}
					case ContextType.CONTEXT_MIMEVERSION_HEADER_VALUE: // mime-version: ...
					{
						bool areDone = false;
						StringBuilder sb = new StringBuilder();
						while (!areDone && !sr.EndOfStream)
						{
							char ch = ReadChar();
							switch (ch)
							{
								case '\r': areDone = true; break;
								case ':': t = new Token(Token.TokenType.CHN_COLON); areDone = true; break;
								case ' ': case '\t': SkipLWS(); break;
								default:
									while (((int)ch) > 31 && ((int)ch) < 127)
									{
										ReadChar();
										sb.Append(ch);
										ch = PeekChar();
									}
									break;
							}
						}
						if (t == null)
							t = new Token(Token.TokenType.CHN_HEADERNAME, sb.ToString());
						break;
					}

					case ContextType.CONTEXT_UNSTRUCTUREDTEXT_WITH_ENCODED_WORDS: // subjects, comments: ...
					//
					//
					//
					{
						bool areDone = false;
						StringBuilder sb = new StringBuilder();
						while (!areDone && !sr.EndOfStream)
						{
							char ch = PeekChar();
							switch (ch)
							{
								case '\r': areDone = true; break;
								case '@': ReadChar(); t = new Token(Token.TokenType.CAHV_AT); break;
								case '.': ReadChar(); t = new Token(Token.TokenType.CAHV_PERIOD); break;
								case ',': ReadChar(); t = new Token(Token.TokenType.CAHV_COMMA); break;
								case ':': ReadChar(); t = new Token(Token.TokenType.CAHV_COLON); break;
								case ';': ReadChar(); t = new Token(Token.TokenType.CAHV_SEMICOLON); break;
								case '(': ParseComment(Token.TokenType.CAHV_LWS); break;
								case '[': t = ParseDomainLiteral(); break;
								case '<': t = ParseRoutingInformation(); break;
								case '"': t = ParseQuotedString(Token.TokenType.CAHV_QUOTED_STRING); break;
								case '\t':
								case ' ': SkipLWS(); break;
								default:
									if (0 < (short)ch && (short)ch < 127)
										t = ParseText();
//										t = ParseAtom();
									else
										throw new LexerException(GetMessage("LexerError_UnexpectedCharacter", ch));
									break;
							}
							if (t != null)
								areDone = true;
						}
						break;
					}
					case ContextType.CONTEXT_MESSAGE_BODY: // body all the way to the end or to the boundary(--)
					//
					//
					//
						while(!areDone)
						{
							ParseText();
						}
						break;
					case ContextType.CONTEXT_UNSTRUCTUREDTEXT: // <some_header> : ...
					//
					//
					//
					default:
						throw new LexerException(GetMessage("LexerError_UnexpectedContext", Context));
				}
			return t;
		} // end of NextToken

		public Token PeekNextToken(ContextType newcontext = ContextType.CONTEXT_NONE)
		{
			if (newcontext != ContextType.CONTEXT_NONE)
				Context = newcontext;

			Token t;
			if (lookaheadTokens.Count > 0)
			{
				Tuple<Token, ContextType> tuple = lookaheadTokens.Peek();
				if (tuple.Item2 != Context)
					throw new LexerException(GetMessage("LexerError_AttemptToProcessLookaheadTokensRetrievedOnDifferentContext", tuple.Item1.TType, Context, tuple.Item2));
				t = tuple.Item1;
			}
			else
			{
				t = NextToken();
				lookaheadTokens.Enqueue(new Tuple<Token, ContextType>(t, Context));
			}
			return t;
		} // end of PeekNextToken

		public Token AssertNextTokenInContextIs(ContextType newcontext = ContextType.CONTEXT_NONE, params Token.TokenType[] tts)
		{
			Token t = NextToken(newcontext);
			if (tts != null)
				if (!tts.Contains<Token.TokenType>(t.TType))
				{
					StringBuilder sb = new StringBuilder();
					foreach (Token.TokenType tt in tts)
						sb.Append(" " + tt.ToString());
					sb.Remove(0, 1);
					throw new LexerException(GetMessage("LexerError_UnexpectedTokenInContextButExpected", t, Context, sb.ToString()));
				}
			return t;
		}

		public Token AssertNextTokenIs(params Token.TokenType[] tts)
		{
			Token t = NextToken();
			if (tts != null)
				if (!tts.Contains<Token.TokenType>(t.TType))
				{
					StringBuilder sb = new StringBuilder();
					foreach (Token.TokenType tt in tts)
						sb.Append(" " + tt.ToString());
					sb.Remove(0, 1);
					throw new LexerException(GetMessage("LexerError_UnexpectedTokenInContextButExpected", t, Context, sb.ToString()));
				}
			return t;
		}

		public void AssertThisTokenIs(Token t, params Token.TokenType[] tts)
		{
			if (tts != null)
				if (!tts.Contains<Token.TokenType>(t.TType))
				{
					StringBuilder sb = new StringBuilder();
					foreach (Token.TokenType tt in tts)
						sb.Append(" " + tt.ToString());
					sb.Remove(0, 1);
					throw new LexerException(GetMessage("LexerError_UnexpectedTokenInContextButExpected", t, Context, sb.ToString()));
				}
		}
	}
}
